
import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt
from PIL import Image
import re
import matplotlib.patches as patches
!pip install ultralytics -qq
from ultralytics import YOLO
import os
import yaml
from sklearn.model_selection import train_test_split


train=pd.read_csv('/kaggle/input/Train (11).csv')


# normalizes the ToothClass values into a smaller
# 11-18 becomes 0-7
# 21-28 becomes 8-15
# 31-38 becomes 16-23
# 41-48 becomes 24à
train.loc[train.ToothClass.isin(range(11,19)),'ToothClass']=train.loc[train.ToothClass.isin(range(11,19)),'ToothClass'].map(lambda x:x-11)
train.loc[train.ToothClass.isin(range(21,29)),'ToothClass']=train.loc[train.ToothClass.isin(range(21,29)),'ToothClass'].map(lambda x:x-13)
train.loc[train.ToothClass.isin(range(31,39)),'ToothClass']=train.loc[train.ToothClass.isin(range(31,39)),'ToothClass'].map(lambda x:x-15)
train.loc[train.ToothClass.isin(range(41,49)),'ToothClass']=train.loc[train.ToothClass.isin(range(41,49)),'ToothClass'].map(lambda x:x-17)


def conv(x):
#     """conv Function: Converts a numeric ID to a zero-padded string and creates a full file path string."""
    a=(6-len(str(x)))*'0'+str(x)
    return f'/kaggle/input/train (1)/train/ID_{a}.png'
def cmpt_id(x):
#      '''Converts a numeric ID to a zero-padded string prefixed with 'ID_'''
    a=(6-len(str(x)))*'0'+str(x)
    return f'ID_{a}'
train['file_path']=train['Image_ID'].map(conv)


random_seed = 42
np.random.seed(random_seed)

unique_image_ids = train['Image_ID'].unique()
shuffled_indices = np.random.permutation(len(unique_image_ids))
unique_image_ids = unique_image_ids[shuffled_indices]


unique_image_ids

array([1178,  865,  101, ..., 1130,  860, 1126])


num_validation = int(0.1* len(unique_image_ids))
validation_ids = unique_image_ids[:num_validation]
training_ids = unique_image_ids[num_validation:]
val_df =train[train['Image_ID'].isin(validation_ids)]
train_df =train[train['Image_ID'].isin(training_ids)]


train_df.reset_index(drop=True)
val_df.reset_index(drop=True)


len(train_df.Image_ID.unique())

1080


train_df['Image_ID']=train_df['Image_ID'].map(cmpt_id)
val_df['Image_ID']=val_df['Image_ID'].map(cmpt_id)

/tmp/ipykernel_34/2645421140.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  train_df['Image_ID']=train_df['Image_ID'].map(cmpt_id)
/tmp/ipykernel_34/2645421140.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  val_df['Image_ID']=val_df['Image_ID'].map(cmpt_id)


import shutil


copy_from_folder = "/kaggle/input/train (1)/train"


def create_data_folder(destination_folder, df):
    if not os.path.exists(destination_folder):
        os.makedirs(destination_folder)
    for img_path in df['file_path'].unique():
        shutil.copy(img_path, destination_folder)


train_destination_folder = "/kaggle/working/train/images"
val_destination_folder = "/kaggle/working/val/images"


create_data_folder(train_destination_folder, train_df)
create_data_folder(val_destination_folder, val_df)


from tqdm import tqdm


train


def convert_bbox_to_string(bbox):    
    xmin, ymin, xmax, ymax = bbox
    #Calculate the Center X and Y Coordinates 
    x_center = (xmin + xmax) /1024
    y_center = (ymin + ymax) /1024
    #Calculate the Width and Height
    width = (xmax - xmin)/512
    height = (ymax - ymin)/512  
    #Format the Box Values
    box = [x_center, y_center,width,height]
    box = [f"{i:.4g}" for i in box]
    return " ".join(box)


def create_label_folder(label_dir, dataframe):
    """This function creates label files for object detection tasks.
    Each label file contains annotations for a specific image, including class IDs and bounding box coordinates."""
    if not os.path.exists(label_dir):
        os.makedirs(label_dir)
        
    unique_images = dataframe['Image_ID'].unique()
    
    for i,img in zip(tqdm(range(len(unique_images))), unique_images):
        df = dataframe[dataframe['Image_ID'] == img]
        output_lines = []
        for index, row in df.iterrows():
            category_id = int(row['ToothClass'])

            bbox_string = convert_bbox_to_string(row[['Xmin','Ymin','Xmax','Ymax']])
            output_lines.append(f'{category_id} {bbox_string}')

        with open(f"{label_dir}/{img}.txt", 'w') as f:
            for line in output_lines:
                f.write(line + '\n')


create_label_folder("/kaggle/working/train/labels", train_df)
create_label_folder("/kaggle/working/val/labels", val_df)

100%|██████████| 1080/1080 [00:10<00:00, 101.56it/s]
100%|██████████| 120/120 [00:00<00:00, 137.56it/s]


num_classes = 32
names = []
for i in range(32):
    names.append(f'tooth{i}')


train_dest_dir = "/kaggle/working/train"
val_dest_dir = "/kaggle/working/val"


dict_file = {
    'train': train_dest_dir,
    'val' : val_dest_dir,
    'nc': num_classes,
    'names': names
    }

with open("/kaggle/working/data.yaml", 'w+') as file:
    yaml.dump(dict_file, file)


### read yaml file created
def read_yaml_file(file_path):
    with open(file_path, 'r') as file:
        try:
            data = yaml.safe_load(file)
            return data
        except yaml.YAMLError as e:
            print("Error reading YAML:", e)
            return None

### print it with newlines
def print_yaml_data(data):
    formatted_yaml = yaml.dump(data, default_style=False)
    print(formatted_yaml)

yaml_data = read_yaml_file("/kaggle/working/data.yaml")

if yaml_data:
    print_yaml_data(yaml_data)

names:
- tooth0
- tooth1
- tooth2
- tooth3
- tooth4
- tooth5
- tooth6
- tooth7
- tooth8
- tooth9
- tooth10
- tooth11
- tooth12
- tooth13
- tooth14
- tooth15
- tooth16
- tooth17
- tooth18
- tooth19
- tooth20
- tooth21
- tooth22
- tooth23
- tooth24
- tooth25
- tooth26
- tooth27
- tooth28
- tooth29
- tooth30
- tooth31
nc: 32
train: /kaggle/working/train
val: /kaggle/working/val


#configure weights and biases
import wandb

try:
    from kaggle_secrets import UserSecretsClient
    user_secrets = UserSecretsClient()
#     api_key = user_secrets.get_secret("wandb_api")
    wandb.login(key='b9fde7760be6d0b70d2f0d4c1b3acce034808c13')
    anony = None
except:
    anony = "must"
    print('If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \nGet your W&B access token from here: https://wandb.ai/authorize')

wandb: W&B API key is configured. Use `wandb login --relogin` to force relogin
wandb: WARNING If you're specifying your api key in code, ensure this code is not shared publicly.
wandb: WARNING Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.
wandb: Appending key for api.wandb.ai to your netrc file: /root/.netrc


from ultralytics import YOLO
model = YOLO('yolov8n.pt')

Downloading https://github.com/ultralytics/assets/releases/download/v8.2.0/yolov8n.pt to 'yolov8n.pt'...

100%|██████████| 6.23M/6.23M [00:00<00:00, 77.1MB/s]


EPOCHS = 30
BATCH_SIZE = 32
OPTIMIZER ='AdamW'
SEED = 42
NAME = f"Arm_Yolo_{EPOCHS}"
DEVICE = 'cuda'
VERBOSE = False
RESUME = False
PATIENCE = 10


%%time
### train
model.train(
    data = "/kaggle/working/data.yaml",
    task = 'detect',
    imgsz = (512,512),
    epochs = EPOCHS,
    batch = BATCH_SIZE,
    optimizer = OPTIMIZER,
    patience = PATIENCE,
    name = NAME,
    seed = SEED,
    val = True,
    resume = RESUME,
    device = DEVICE,
    fliplr=0.0,
    label_smoothing=0.05,
    close_mosaic=12,
    cos_lr=True,
    degrees=15,
    verbose = VERBOSE
)

Ultralytics YOLOv8.2.18 🚀 Python-3.10.13 torch-2.1.2 CUDA:0 (Tesla P100-PCIE-16GB, 16276MiB)
engine/trainer: task=detect, mode=train, model=yolov8n.pt, data=/kaggle/working/data.yaml, epochs=30, time=None, patience=10, batch=32, imgsz=(512, 512), save=True, save_period=-1, cache=False, device=cuda, workers=8, project=None, name=Arm_Yolo_30, exist_ok=False, pretrained=True, optimizer=AdamW, verbose=False, seed=42, deterministic=True, single_cls=False, rect=False, cos_lr=True, close_mosaic=12, resume=False, amp=True, fraction=1.0, profile=False, freeze=None, multi_scale=False, overlap_mask=True, mask_ratio=4, dropout=0.0, val=True, split=val, save_json=False, save_hybrid=False, conf=None, iou=0.7, max_det=300, half=False, dnn=False, plots=True, source=None, vid_stride=1, stream_buffer=False, visualize=False, augment=False, agnostic_nms=False, classes=None, retina_masks=False, embed=None, show=False, save_frames=False, save_txt=False, save_conf=False, save_crop=False, show_labels=True, show_conf=True, show_boxes=True, line_width=None, format=torchscript, keras=False, optimize=False, int8=False, dynamic=False, simplify=False, opset=None, workspace=4, nms=False, lr0=0.01, lrf=0.01, momentum=0.937, weight_decay=0.0005, warmup_epochs=3.0, warmup_momentum=0.8, warmup_bias_lr=0.1, box=7.5, cls=0.5, dfl=1.5, pose=12.0, kobj=1.0, label_smoothing=0.05, nbs=64, hsv_h=0.015, hsv_s=0.7, hsv_v=0.4, degrees=15, translate=0.1, scale=0.5, shear=0.0, perspective=0.0, flipud=0.0, fliplr=0.0, bgr=0.0, mosaic=1.0, mixup=0.0, copy_paste=0.0, auto_augment=randaugment, erasing=0.4, crop_fraction=1.0, cfg=None, tracker=botsort.yaml, save_dir=runs/detect/Arm_Yolo_30
Downloading https://ultralytics.com/assets/Arial.ttf to '/root/.config/Ultralytics/Arial.ttf'...

100%|██████████| 755k/755k [00:00<00:00, 14.5MB/s]
2024-05-19 07:23:24,041	INFO util.py:124 -- Outdated packages:
  ipywidgets==7.7.1 found, needs ipywidgets>=8
Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.
2024-05-19 07:23:24,849	INFO util.py:124 -- Outdated packages:
  ipywidgets==7.7.1 found, needs ipywidgets>=8
Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.

Overriding model.yaml nc=80 with nc=32

                   from  n    params  module                                       arguments                     
  0                  -1  1       464  ultralytics.nn.modules.conv.Conv             [3, 16, 3, 2]                 
  1                  -1  1      4672  ultralytics.nn.modules.conv.Conv             [16, 32, 3, 2]                
  2                  -1  1      7360  ultralytics.nn.modules.block.C2f             [32, 32, 1, True]             
  3                  -1  1     18560  ultralytics.nn.modules.conv.Conv             [32, 64, 3, 2]                
  4                  -1  2     49664  ultralytics.nn.modules.block.C2f             [64, 64, 2, True]             
  5                  -1  1     73984  ultralytics.nn.modules.conv.Conv             [64, 128, 3, 2]               
  6                  -1  2    197632  ultralytics.nn.modules.block.C2f             [128, 128, 2, True]           
  7                  -1  1    295424  ultralytics.nn.modules.conv.Conv             [128, 256, 3, 2]              
  8                  -1  1    460288  ultralytics.nn.modules.block.C2f             [256, 256, 1, True]           
  9                  -1  1    164608  ultralytics.nn.modules.block.SPPF            [256, 256, 5]                 
 10                  -1  1         0  torch.nn.modules.upsampling.Upsample         [None, 2, 'nearest']          
 11             [-1, 6]  1         0  ultralytics.nn.modules.conv.Concat           [1]                           
 12                  -1  1    148224  ultralytics.nn.modules.block.C2f             [384, 128, 1]                 
 13                  -1  1         0  torch.nn.modules.upsampling.Upsample         [None, 2, 'nearest']          
 14             [-1, 4]  1         0  ultralytics.nn.modules.conv.Concat           [1]                           
 15                  -1  1     37248  ultralytics.nn.modules.block.C2f             [192, 64, 1]                  
 16                  -1  1     36992  ultralytics.nn.modules.conv.Conv             [64, 64, 3, 2]                
 17            [-1, 12]  1         0  ultralytics.nn.modules.conv.Concat           [1]                           
 18                  -1  1    123648  ultralytics.nn.modules.block.C2f             [192, 128, 1]                 
 19                  -1  1    147712  ultralytics.nn.modules.conv.Conv             [128, 128, 3, 2]              
 20             [-1, 9]  1         0  ultralytics.nn.modules.conv.Concat           [1]                           
 21                  -1  1    493056  ultralytics.nn.modules.block.C2f             [384, 256, 1]                 
 22        [15, 18, 21]  1    757552  ultralytics.nn.modules.head.Detect           [32, [64, 128, 256]]          
Model summary: 225 layers, 3017088 parameters, 3017072 gradients, 8.2 GFLOPs

Transferred 319/355 items from pretrained weights
TensorBoard: Start with 'tensorboard --logdir runs/detect/Arm_Yolo_30', view at http://localhost:6006/

wandb: Currently logged in as: lassouedaymenla (comp1). Use `wandb login --relogin` to force relogin

Freezing layer 'model.22.dfl.conv.weight'
AMP: running Automatic Mixed Precision (AMP) checks with YOLOv8n...
AMP: checks passed ✅
WARNING ⚠️ updating to 'imgsz=512'. 'train' and 'val' imgsz must be an integer, while 'predict' and 'export' imgsz may be a [h, w] list or an integer, i.e. 'yolo export imgsz=640,480' or 'yolo export imgsz=640'

train: Scanning /kaggle/working/train/labels... 1080 images, 0 backgrounds, 0 corrupt: 100%|██████████| 1080/1080 [00:01<00:00, 599.06it/s]

train: New cache created: /kaggle/working/train/labels.cache

albumentations: Blur(p=0.01, blur_limit=(3, 7)), MedianBlur(p=0.01, blur_limit=(3, 7)), ToGray(p=0.01), CLAHE(p=0.01, clip_limit=(1, 4.0), tile_grid_size=(8, 8))

val: Scanning /kaggle/working/val/labels... 120 images, 0 backgrounds, 0 corrupt: 100%|██████████| 120/120 [00:00<00:00, 397.03it/s]


sample_submission=pd.read_csv('/kaggle/input/SampleSubmission (3).csv')


import json
def get_prediction(model, image_id, phase="val"):
    if phase == "val":
        path = '/kaggle/input/test (1)/test'
    else:
        path = '/kaggle/input/train (1)/train'
    pred = model.predict(os.path.join(path,image_id+'.png'), imgsz=512, conf=0.1, augment=False, agnostic_nms=False)
    pred = json.loads(pred[0].tojson())
    return pred


def f(x):
    if x in(range(0,8)):
        return int(x+11)
    if x in(range(8,16)):
        return int(x+13)
    if x in(range(16,24)):
        return int(x+15)
    if x in(range(24,32)):
        return int(x+17)
    else:return ' '


def generate_prediction(model, ss):
    unique_imgs = ss["Image_ID"].unique()
    transformed_pred = []
    for image_id in tqdm(unique_imgs, total=len(unique_imgs), desc="Generaing predictions"):
        pred = get_prediction(model, image_id, phase="val")
        for item in pred:
            transformed_pred.append({
                'Image_ID': image_id,
                'ToothClass': f(int(item['name'][5:])),
                'confidence': item['confidence'],
                'Xmin': item['box']['x1'],
                'Ymin': item['box']['y1'],
                'Xmax': item['box']['x2'],
                'Ymax': item['box']['y2']
            })
    transformed_pred = pd.DataFrame(transformed_pred)
    return transformed_pred


test=pd.read_csv('/kaggle/input/Test (13).csv')


transformed_pred=generate_prediction(model,test)

Generaing predictions: 100%|██████████| 600/600 [00:16<00:00, 36.29it/s]


transformed_pred["ToothClass"] = "class_" + transformed_pred["ToothClass"].astype(str)
transformed_pred.columns = ["Image_ID", "class", "confidence", "xmin", "ymin", "xmax", "ymax"]


transformed_pred


transformed_pred['class'].value_counts()

class
class_26    320
class_22    319
class_14    316
class_36    313
class_42    312
class_41    311
class_46    310
class_32    309
class_16    308
class_45    308
class_44    307
class_11    307
class_31    307
class_21    305
class_15    305
class_43    304
class_33    304
class_12    304
class_24    303
class_35    303
class_34    301
class_25    301
class_13    296
class_23    294
class_37    262
class_47    245
class_27    224
class_17    217
class_18     18
class_48     18
class_38     15
class_28     14
Name: count, dtype: int64


transformed_pred.to_csv('submission-conf0.1.csv',index=False)


transformed_pred['class'].value_counts()

class
class_26    320
class_22    319
class_14    316
class_36    313
class_42    312
class_41    311
class_46    310
class_32    309
class_16    308
class_45    308
class_44    307
class_11    307
class_31    307
class_21    305
class_15    305
class_43    304
class_33    304
class_12    304
class_24    303
class_35    303
class_34    301
class_25    301
class_13    296
class_23    294
class_37    262
class_47    245
class_27    224
class_17    217
class_18     18
class_48     18
class_38     15
class_28     14
Name: count, dtype: int64

	Image_ID	ToothClass	Xmin	Ymin	Xmax	Ymax	file_path
0	23	25	189	362	231	402	/kaggle/input/train (1)/train/ID_000023.png
1	23	29	82	175	156	254	/kaggle/input/train (1)/train/ID_000023.png
2	23	21	358	169	432	249	/kaggle/input/train (1)/train/ID_000023.png
3	23	17	294	361	335	401	/kaggle/input/train (1)/train/ID_000023.png
4	23	27	127	297	179	347	/kaggle/input/train (1)/train/ID_000023.png
...	...	...	...	...	...	...	...
1583	1191	10	129	285	168	335	/kaggle/input/train (1)/train/ID_001191.png
1584	1191	3	358	237	411	293	/kaggle/input/train (1)/train/ID_001191.png
1585	1191	12	74	179	140	252	/kaggle/input/train (1)/train/ID_001191.png
1586	1191	2	345	282	384	331	/kaggle/input/train (1)/train/ID_001191.png
1587	1191	1	316	330	364	370	/kaggle/input/train (1)/train/ID_001191.png

	Image_ID	ToothClass	Xmin	Ymin	Xmax	Ymax	file_path
0	0	18	284	315	325	361	/kaggle/input/train (1)/train/ID_000000.png
1	0	22	372	119	448	193	/kaggle/input/train (1)/train/ID_000000.png
2	0	30	40	114	113	186	/kaggle/input/train (1)/train/ID_000000.png
3	0	20	325	236	379	287	/kaggle/input/train (1)/train/ID_000000.png
4	0	24	200	362	237	411	/kaggle/input/train (1)/train/ID_000000.png
...	...	...	...	...	...	...	...
16008	1199	2	337	318	388	366	/kaggle/input/train (1)/train/ID_001199.png
16009	1199	1	309	339	349	389	/kaggle/input/train (1)/train/ID_001199.png
16010	1199	9	177	352	218	400	/kaggle/input/train (1)/train/ID_001199.png
16011	1199	11	111	288	174	343	/kaggle/input/train (1)/train/ID_001199.png
16012	1199	10	137	330	188	381	/kaggle/input/train (1)/train/ID_001199.png

lr/pg0	█▅▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
lr/pg1	▃▆████▇▇▇▇▆▆▆▅▅▅▄▄▃▃▃▂▂▂▂▁▁▁▁▁
lr/pg2	▃▆████▇▇▇▇▆▆▆▅▅▅▄▄▃▃▃▂▂▂▂▁▁▁▁▁
metrics/mAP50(B)	▁▂▁▄▄▁▁▆▆▇▂▅▇▇█▇▆▇████████████
metrics/mAP50-95(B)	▁▂▁▃▃▁▁▄▄▅▂▃▄▅▅▅▅▇▇▇██▇███████
metrics/precision(B)	▂▃▅▅▆▁▃▆▆█▄▅▇▇▇▇▅██▇▇▇████████
metrics/recall(B)	▁▂▁▄▄▁▁▅▆▇▃▄▇▆█▆▆▇▇▇██████████
model/GFLOPs	▁
model/parameters	▁
model/speed_PyTorch(ms)	▁
train/box_loss	█▆▆▆▅▅▅▅▅▅▄▅▄▄▄▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁
train/cls_loss	█▄▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁
train/dfl_loss	█▅▅▅▅▄▄▄▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁▁
val/box_loss	█▆▅▅▅█▅▅▅▄▆▅▅▄▆▄▂▁▂▁▁▁▂▁▁▁▁▁▁▁
val/cls_loss	▅▅▃▃▇█▂▂▁▄▃▂▁▁▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁
val/dfl_loss	█▃▃▂▂▅▂▂▂▂▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁

lr/pg0	0.00013
lr/pg1	0.00013
lr/pg2	0.00013
metrics/mAP50(B)	0.99219
metrics/mAP50-95(B)	0.90805
metrics/precision(B)	0.95769
metrics/recall(B)	0.94589
model/GFLOPs	8.227
model/parameters	3017088
model/speed_PyTorch(ms)	2.374
train/box_loss	0.4919
train/cls_loss	0.34235
train/dfl_loss	0.8567
val/box_loss	0.40051
val/cls_loss	0.30524
val/dfl_loss	0.80803

	Image_ID	class	confidence	xmin	ymin	xmax	ymax
0	ID_001200	class_35	0.94939	344.53448	228.70889	396.51727	279.23587
1	ID_001200	class_46	0.93689	82.71478	160.39441	156.75211	241.95648
2	ID_001200	class_47	0.93645	61.27040	104.65893	128.32320	172.63263
3	ID_001200	class_36	0.93530	350.89240	157.68445	421.90753	237.52722
4	ID_001200	class_44	0.93490	131.95337	272.65680	180.60272	321.44830
...	...	...	...	...	...	...	...
8375	ID_001799	class_14	0.85651	313.80225	272.86542	373.51965	325.82635
8376	ID_001799	class_15	0.84385	334.92413	230.42412	395.79706	284.67377
8377	ID_001799	class_12	0.82248	291.01959	315.86652	342.56622	365.41693
8378	ID_001799	class_11	0.68745	253.75346	341.51059	311.87579	395.75760
8379	ID_001799	class_14	0.15729	334.34015	230.59062	395.34357	284.69769

Run history:

Run summary: